package com.biqee.service.crawler.impl;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.log4j.Logger;

import com.biqee.service.crawler.PageFetcher;
import com.biqee.util.IOCloseUtil;

public class PageFetcherImpl implements PageFetcher {
	private static final Logger logger = Logger.getLogger(PageFetcherImpl.class);
	private static final String DEFAULT_ENCODE = "GBK";
	
	@Override
	public String getAsStringWithSpiderHeader(String url, String encode) {
		HttpClient httpClient = createSpiderHttpClient();
		
		return queryAsString(httpClient, url, encode);
	}

	@Override
	public String getAsStringWithBrowserHeader(String url, String encode) {
		HttpClient httpClient = createBrowserHttpClient();
		
		return queryAsString(httpClient, url, encode);
	}

	/*
	 	BAIDU   "Baiduspider+(+http://www.baidu.com/search/spider.htm)"
		GOOGLE  "Googlebot/2.1+(+http://www.google.com/bot.html)"
		GOOGLE  "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
		SOSO    "Sosospider+(+http://help.soso.com/webspider.htm)"
		YAHOO   "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)"
		YAHOO   "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)"
		YAHOO   "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
		SOGOU   "Sogou inst spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
		SOGOU   "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
		BING    "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)"
		BING    "msnbot/2.0b (+http://search.msn.com/msnbot.htm)"
	 */
	private HttpClient createSpiderHttpClient() {
		Header header = new Header("User-Agent", "iSeacher+(+http://www.iSeacher.com/bot.html)");
		return createHttpClient(header);
	}
	
	private HttpClient createBrowserHttpClient() {
		Header header = new Header("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");
		return createHttpClient(header);
	}
	
	private HttpClient createHttpClient(Header header) {
		List<Header> headers = new ArrayList<Header>();
		headers.add(header);
	
		HttpClient httpClient = new HttpClient();
		httpClient.getHostConfiguration().getParams()
				.setParameter("http.default-headers", headers);
		
		return httpClient;
	}

	private String queryAsString(HttpClient httpClient, String url, String encode) {
		if(encode == null) {
			encode = DEFAULT_ENCODE;
		}
		
		HttpMethod method = new GetMethod(url);
		method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, encode);
		InputStream in = null;
		
		try {
			httpClient.executeMethod(method);
			int statusCode = method.getStatusCode();
			logger.info("get: " + url + ", statusCode: " + statusCode);
			in = method.getResponseBodyAsStream();
		} catch (Exception e) {
			logger.error("get content error for: " + url, e);
			return "";
		}
		
		StringBuffer buffer = new StringBuffer();
		Reader reader = null;
		InputStreamReader isr = null;
		
	    try {
	    	isr = new InputStreamReader(in, encode);
	    	reader = new BufferedReader(isr);
	    	int ch;
	    	while ((ch = reader.read()) > -1) {
	    		buffer.append((char)ch);
	    	}
	    } catch (Exception ex) {
	    	logger.error("read content failed. url: " + url, ex);
	    } finally {
	    	IOCloseUtil.safeClose(reader);
	    	IOCloseUtil.safeClose(isr);
	    	IOCloseUtil.safeClose(in);
	    }
	    
	    return buffer.toString();
	}
	
//	private static final Logger logger = Logger
//			.getLogger(PageFetcherImpl.class);
//	private static final String DEFAULT_ENCODE = "GBK";
//	
//	private HttpClient createHttpClient() {
//		List<Header> headers = new ArrayList<Header>();
//		headers.add(new Header("User-Agent",
//				"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)"));
//
//		HttpClient httpClient = new HttpClient();
//		httpClient.getHostConfiguration().getParams()
//				.setParameter("http.default-headers", headers);
//		
//		return httpClient;
//	}
//	
//	@Override
//	public String getAsStringByUrlWithPseudoBrowserHeader(String url) {
//		return getAsStringByUrlWithPseudoBrowserHeader(url, DEFAULT_ENCODE);
//	}
//
//	@Override
//	public String getAsStringByUrlWithPseudoBrowserHeader(String url,
//			String encode) {
//		if(encode == null) {
//			encode = DEFAULT_ENCODE;
//		}
//		
//		HttpMethod method = new GetMethod(url);
//		method.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, encode);
//		InputStream in = null;
//		HttpClient httpClient = createHttpClient();
//		try {
//			httpClient.executeMethod(method);
//			int statusCode = method.getStatusCode();
//			logger.info("get: " + url + ", statusCode: " + statusCode);
//			in = method.getResponseBodyAsStream();
//		} catch (Exception e) {
//			logger.error("get content error for: " + url, e);
//			return "";
//		}
//		
//		StringBuffer buffer = new StringBuffer();
//		Reader reader = null;
//		InputStreamReader isr = null;
//		
//	    try {
//	    	isr = new InputStreamReader(in, encode);
//	    	reader = new BufferedReader(isr);
//	    	int ch;
//	    	while ((ch = reader.read()) > -1) {
//	    		buffer.append((char)ch);
//	    	}
//	    } catch (Exception ex) {
//	    	logger.error("read content failed. url: " + url, ex);
//	    } finally {
//	    	IOCloseUtil.safeClose(reader);
//	    	IOCloseUtil.safeClose(isr);
//	    	IOCloseUtil.safeClose(in);
//	    }
//	    
//	    return buffer.toString();
//	}



}
